package com.examples.spider.processor;

import org.assertj.core.util.Lists;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.pipeline.ConsolePipeline;
import us.codecraft.webmagic.pipeline.FilePipeline;
import us.codecraft.webmagic.pipeline.Pipeline;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.scheduler.QueueScheduler;
import us.codecraft.webmagic.scheduler.component.HashSetDuplicateRemover;
import us.codecraft.webmagic.selector.Selectable;

public class BookChapterProcessor implements PageProcessor {

    public void process(Page page) {

//        String chapterName = page.getHtml().xpath("//*[@id='wrapper']").xpath("//div[@class='bookname']/h1/text()").toString();
        String chapterName = page.getHtml().xpath("///html/body/div[3]/div[1]/div[2]/div[1]/h1/text()").toString();
        page.putField("chapterName", chapterName);
        Selectable contentNode = page.getHtml().xpath("//*[@id='content']/text()");
        page.putField("chapterContent", contentNode.toString());
        String aa = contentNode.toString().replace("    ", "\n");
        System.out.println(aa);
        //page.addTargetRequests(page.getHtml().links().regex("(https://www.xxbiquge.com/31_31906/[\\d]+.html)").all());
    }

    public Site getSite() {
        Site site = Site.me().setRetryTimes(2).setSleepTime(100);
        return site;
    }

    public static void main(String[] args) {
/*        Pipeline filePipeline = new FilePipeline("D:/灵山.txt");
        Spider.create(new BookChapterProcessor()).addUrl("https://www.xxbiquge.com/31_31906/1968571.html")
                .setScheduler(new QueueScheduler().setDuplicateRemover(new HashSetDuplicateRemover()))
                .setPipelines(Lists.newArrayList(filePipeline, new ConsolePipeline()))
                .thread(5)
                .run();*/
        Pipeline filePipeline = new FilePipeline("D:/征服异界从网游开始.txt");
        Spider.create(new BookChapterProcessor()).addUrl("http://www.boquku.com/book/48571/")
                .setScheduler(new QueueScheduler().setDuplicateRemover(new HashSetDuplicateRemover()))
                .setPipelines(Lists.newArrayList(filePipeline, new ConsolePipeline()))
                .thread(5)
                .run();
    }


}
